{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3b70cb38-a0a0-4f0a-9d77-eafa92301338",
   "metadata": {},
   "source": [
    "### Seldon V2 Batch Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3d069ec2-50a0-4fd3-b65e-d178cb7b142d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"NAMESPACE\"] = \"seldon-mesh\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "93d79639-2651-4b83-bbcf-404e772f1f3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'172.18.255.2'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MESH_IP=!kubectl get svc seldon-mesh -n ${NAMESPACE} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'\n",
    "MESH_IP=MESH_IP[0]\n",
    "import os\n",
    "os.environ['MESH_IP'] = MESH_IP\n",
    "MESH_IP"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1098488-4929-4df4-b3f9-8e448e87bcec",
   "metadata": {},
   "source": [
    "## Deploy Models and Pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "77169ee9-896f-4ee3-b154-b79997d71925",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn\"\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n",
      "  memory: 100Ki\r\n"
     ]
    }
   ],
   "source": [
    "!cat models/sklearn-iris-gs.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e5653b7f-b8f5-4f08-ac2c-a26775ddf027",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Pipeline\r\n",
      "metadata:\r\n",
      "  name: iris-pipeline\r\n",
      "spec:\r\n",
      "  steps:\r\n",
      "    - name: iris\r\n",
      "  output:\r\n",
      "    steps:\r\n",
      "    - iris\r\n"
     ]
    }
   ],
   "source": [
    "!cat pipelines/iris.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3c51fc8b-3164-4e1a-8616-452f62d61d22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: tfsimple1\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/triton/simple\"\r\n",
      "  requirements:\r\n",
      "  - tensorflow\r\n",
      "  memory: 100Ki\r\n"
     ]
    }
   ],
   "source": [
    "!cat models/tfsimple1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "992a0404-e1d6-4d55-a963-2cdebd42231c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Pipeline\r\n",
      "metadata:\r\n",
      "  name: tfsimple\r\n",
      "spec:\r\n",
      "  steps:\r\n",
      "    - name: tfsimple1\r\n",
      "  output:\r\n",
      "    steps:\r\n",
      "    - tfsimple1\r\n"
     ]
    }
   ],
   "source": [
    "!cat pipelines/tfsimple.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b4f7affb-5418-4209-b665-4478482ea3dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/iris created\n",
      "pipeline.mlops.seldon.io/iris-pipeline created\n",
      "model.mlops.seldon.io/tfsimple1 created\n",
      "pipeline.mlops.seldon.io/tfsimple created\n"
     ]
    }
   ],
   "source": [
    "!kubectl apply -f models/sklearn-iris-gs.yaml -n ${NAMESPACE}\n",
    "!kubectl apply -f pipelines/iris.yaml -n ${NAMESPACE}\n",
    "\n",
    "!kubectl apply -f models/tfsimple1.yaml -n ${NAMESPACE}\n",
    "!kubectl apply -f pipelines/tfsimple.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "66b6b1f3-08bc-4212-be47-9a1df0b18348",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/iris condition met\n",
      "model.mlops.seldon.io/tfsimple1 condition met\n",
      "pipeline.mlops.seldon.io/iris-pipeline condition met\n",
      "pipeline.mlops.seldon.io/tfsimple condition met\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n ${NAMESPACE}\n",
    "!kubectl wait --for condition=ready --timeout=300s pipelines --all -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c8e501b-fd01-47f6-a211-7de81942d754",
   "metadata": {},
   "source": [
    "## Test Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5b2678d3-a5cb-4b41-ad5b-3f29a7a1202d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"iris_1\",\r\n",
      "  \"model_version\": \"1\",\r\n",
      "  \"id\": \"25e1c1b9-a20f-456d-bdff-c75d5ba83b1f\",\r\n",
      "  \"parameters\": {},\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"predict\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        1\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT64\",\r\n",
      "      \"parameters\": {\r\n",
      "        \"content_type\": \"np\"\r\n",
      "      },\r\n",
      "      \"data\": [\r\n",
      "        2\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris --inference-host ${MESH_IP}:80 \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "31701fb2-1185-415c-a0f8-b05ab89ae2d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"data\": [\r\n",
      "        2\r\n",
      "      ],\r\n",
      "      \"name\": \"predict\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        1\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT64\",\r\n",
      "      \"parameters\": {\r\n",
      "        \"content_type\": \"np\"\r\n",
      "      }\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon pipeline infer iris-pipeline --inference-host ${MESH_IP}:80 \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' |  jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e7fd9ec3-fd25-48ad-bf5d-90231805cd01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"tfsimple1_1\",\r\n",
      "  \"model_version\": \"1\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT0\",\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"data\": [\r\n",
      "        2,\r\n",
      "        4,\r\n",
      "        6,\r\n",
      "        8,\r\n",
      "        10,\r\n",
      "        12,\r\n",
      "        14,\r\n",
      "        16,\r\n",
      "        18,\r\n",
      "        20,\r\n",
      "        22,\r\n",
      "        24,\r\n",
      "        26,\r\n",
      "        28,\r\n",
      "        30,\r\n",
      "        32\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer tfsimple1 --inference-host ${MESH_IP}:80 \\\n",
    "  '{\"outputs\":[{\"name\":\"OUTPUT0\"}], \"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9473915b-0ec4-457c-b304-1ba50e9cd78d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"data\": [\r\n",
      "        2,\r\n",
      "        4,\r\n",
      "        6,\r\n",
      "        8,\r\n",
      "        10,\r\n",
      "        12,\r\n",
      "        14,\r\n",
      "        16,\r\n",
      "        18,\r\n",
      "        20,\r\n",
      "        22,\r\n",
      "        24,\r\n",
      "        26,\r\n",
      "        28,\r\n",
      "        30,\r\n",
      "        32\r\n",
      "      ],\r\n",
      "      \"name\": \"OUTPUT0\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\"\r\n",
      "    },\r\n",
      "    {\r\n",
      "      \"data\": [\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0\r\n",
      "      ],\r\n",
      "      \"name\": \"OUTPUT1\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\"\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon pipeline infer tfsimple --inference-host ${MESH_IP}:80 \\\n",
    "  '{\"outputs\":[{\"name\":\"OUTPUT0\"}], \"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aecd8d62-2802-4e1d-bd1e-78978f3d4972",
   "metadata": {},
   "source": [
    "## MLServer Iris Batch Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9e9bee26-3c1c-44b0-9889-587767cf933e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"inputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"predict\",\r\n",
      "      \"data\": [\r\n",
      "        0.38606369295833043,\r\n",
      "        0.006894049558299753,\r\n",
      "        0.6104082981607108,\r\n",
      "        0.3958954239450676\r\n",
      "      ],\r\n",
      "      \"datatype\": \"FP64\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        4\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!cat batch-inputs/iris-input.txt | head -n 1 | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a3b60a8f-80b5-4803-82b9-9d68605480d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-30 11:05:32,389 [mlserver] INFO - server url: 172.18.255.2\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - model name: iris\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - request headers: {}\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - input file path: batch-inputs/iris-input.txt\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - output file path: /tmp/iris-output.txt\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - workers: 5\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - retries: 3\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - batch interval: 0.0\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - batch jitter: 0.0\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - connection timeout: 60\n",
      "2023-06-30 11:05:32,389 [mlserver] INFO - micro-batch size: 1\n",
      "2023-06-30 11:05:32,503 [mlserver] INFO - Finalizer: processed instances: 100\n",
      "2023-06-30 11:05:32,503 [mlserver] INFO - Total processed instances: 100\n",
      "2023-06-30 11:05:32,503 [mlserver] INFO - Time taken: 0.11 seconds\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "mlserver infer -u ${MESH_IP} -m iris -i batch-inputs/iris-input.txt -o /tmp/iris-output.txt --workers 5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d3b7ef85-6435-4041-9857-2f843091118a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-30 11:05:35,857 [mlserver] INFO - server url: 172.18.255.2\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - model name: iris-pipeline.pipeline\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - request headers: {}\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - input file path: batch-inputs/iris-input.txt\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - output file path: /tmp/iris-pipeline-output.txt\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - workers: 5\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - retries: 3\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - batch interval: 0.0\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - batch jitter: 0.0\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - connection timeout: 60\n",
      "2023-06-30 11:05:35,858 [mlserver] INFO - micro-batch size: 1\n",
      "2023-06-30 11:05:36,145 [mlserver] INFO - Finalizer: processed instances: 100\n",
      "2023-06-30 11:05:36,146 [mlserver] INFO - Total processed instances: 100\n",
      "2023-06-30 11:05:36,146 [mlserver] INFO - Time taken: 0.29 seconds\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "mlserver infer -u ${MESH_IP} -m iris-pipeline.pipeline -i batch-inputs/iris-input.txt -o /tmp/iris-pipeline-output.txt --workers 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2fd223f2-9b91-47c2-99a6-e0f16879b44e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"iris_1\",\r\n",
      "  \"model_version\": \"1\",\r\n",
      "  \"id\": \"46bdfca2-8805-4a72-b1ce-95e4f38c1a19\",\r\n",
      "  \"parameters\": {\r\n",
      "    \"inference_id\": \"46bdfca2-8805-4a72-b1ce-95e4f38c1a19\",\r\n",
      "    \"batch_index\": 0\r\n",
      "  },\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"predict\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        1\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT64\",\r\n",
      "      \"parameters\": {\r\n",
      "        \"content_type\": \"np\"\r\n",
      "      },\r\n",
      "      \"data\": [\r\n",
      "        1\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!cat /tmp/iris-output.txt | head -n 1 | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c14358ae-6206-4f67-84e6-2baa6a54c7db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;39m{\r\n",
      "  \u001b[0m\u001b[34;1m\"model_name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\r\n",
      "  \u001b[0m\u001b[34;1m\"id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"37e8c013-b348-41e8-89b9-fea86a4f9632\"\u001b[0m\u001b[1;39m,\r\n",
      "  \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\r\n",
      "    \u001b[0m\u001b[34;1m\"batch_index\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1\u001b[0m\u001b[1;39m\r\n",
      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\r\n",
      "  \u001b[0m\u001b[34;1m\"outputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\r\n",
      "    \u001b[1;39m{\r\n",
      "      \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"predict\"\u001b[0m\u001b[1;39m,\r\n",
      "      \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\r\n",
      "        \u001b[0;39m1\u001b[0m\u001b[1;39m,\r\n",
      "        \u001b[0;39m1\u001b[0m\u001b[1;39m\r\n",
      "      \u001b[1;39m]\u001b[0m\u001b[1;39m,\r\n",
      "      \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT64\"\u001b[0m\u001b[1;39m,\r\n",
      "      \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\r\n",
      "        \u001b[0m\u001b[34;1m\"content_type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"np\"\u001b[0m\u001b[1;39m\r\n",
      "      \u001b[1;39m}\u001b[0m\u001b[1;39m,\r\n",
      "      \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\r\n",
      "        \u001b[0;39m1\u001b[0m\u001b[1;39m\r\n",
      "      \u001b[1;39m]\u001b[0m\u001b[1;39m\r\n",
      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\r\n",
      "  \u001b[1;39m]\u001b[0m\u001b[1;39m\r\n",
      "\u001b[1;39m}\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!cat /tmp/iris-pipeline-output.txt | head -n 1 | jq ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b881fc4f-6e57-4164-9d8d-d174662a43ec",
   "metadata": {},
   "source": [
    "## Triton TFSimple Batch Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5ae725ef-bf07-42f7-8c57-d4e99ae99a87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"inputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"INPUT0\",\r\n",
      "      \"data\": [\r\n",
      "        75,\r\n",
      "        39,\r\n",
      "        9,\r\n",
      "        44,\r\n",
      "        32,\r\n",
      "        97,\r\n",
      "        99,\r\n",
      "        40,\r\n",
      "        13,\r\n",
      "        27,\r\n",
      "        25,\r\n",
      "        36,\r\n",
      "        18,\r\n",
      "        77,\r\n",
      "        62,\r\n",
      "        60\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ]\r\n",
      "    },\r\n",
      "    {\r\n",
      "      \"name\": \"INPUT1\",\r\n",
      "      \"data\": [\r\n",
      "        39,\r\n",
      "        7,\r\n",
      "        14,\r\n",
      "        58,\r\n",
      "        13,\r\n",
      "        88,\r\n",
      "        98,\r\n",
      "        66,\r\n",
      "        97,\r\n",
      "        57,\r\n",
      "        49,\r\n",
      "        3,\r\n",
      "        49,\r\n",
      "        63,\r\n",
      "        37,\r\n",
      "        12\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!cat batch-inputs/tfsimple-input.txt | head -n 1 | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5c5f360a-0c2c-4759-b30d-6e907aa5c75e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-30 11:22:52,662 [mlserver] INFO - server url: 172.18.255.2\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - model name: tfsimple1\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - request headers: {}\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - input file path: batch-inputs/tfsimple-input.txt\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - output file path: /tmp/tfsimple-output.txt\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - workers: 5\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - retries: 3\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - batch interval: 0.0\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - batch jitter: 0.0\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - connection timeout: 60\n",
      "2023-06-30 11:22:52,662 [mlserver] INFO - micro-batch size: 1\n",
      "2023-06-30 11:22:52,755 [mlserver] INFO - Finalizer: processed instances: 100\n",
      "2023-06-30 11:22:52,755 [mlserver] INFO - Total processed instances: 100\n",
      "2023-06-30 11:22:52,756 [mlserver] INFO - Time taken: 0.09 seconds\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "mlserver infer -u ${MESH_IP} -m tfsimple1 -i batch-inputs/tfsimple-input.txt -o /tmp/tfsimple-output.txt --workers 5 -b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "753bc0ad-2fa0-47d8-a5cc-428cba326806",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-30 11:22:54,065 [mlserver] INFO - server url: 172.18.255.2\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - model name: tfsimple.pipeline\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - request headers: {}\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - input file path: batch-inputs/tfsimple-input.txt\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - output file path: /tmp/tfsimple-pipeline-output.txt\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - workers: 5\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - retries: 3\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - batch interval: 0.0\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - batch jitter: 0.0\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - connection timeout: 60\n",
      "2023-06-30 11:22:54,065 [mlserver] INFO - micro-batch size: 1\n",
      "2023-06-30 11:22:54,302 [mlserver] INFO - Finalizer: processed instances: 100\n",
      "2023-06-30 11:22:54,302 [mlserver] INFO - Total processed instances: 100\n",
      "2023-06-30 11:22:54,303 [mlserver] INFO - Time taken: 0.24 seconds\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "mlserver infer -u ${MESH_IP} -m tfsimple.pipeline -i batch-inputs/tfsimple-input.txt -o /tmp/tfsimple-pipeline-output.txt --workers 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0f8a3641-c8a3-41c5-af4b-2647655fb10d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"tfsimple1_1\",\r\n",
      "  \"model_version\": \"1\",\r\n",
      "  \"id\": \"19952272-b023-4079-aa08-f1880ded05e5\",\r\n",
      "  \"parameters\": {\r\n",
      "    \"inference_id\": \"19952272-b023-4079-aa08-f1880ded05e5\",\r\n",
      "    \"batch_index\": 1\r\n",
      "  },\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT0\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"parameters\": {},\r\n",
      "      \"data\": [\r\n",
      "        115,\r\n",
      "        69,\r\n",
      "        97,\r\n",
      "        112,\r\n",
      "        73,\r\n",
      "        106,\r\n",
      "        58,\r\n",
      "        182,\r\n",
      "        114,\r\n",
      "        66,\r\n",
      "        64,\r\n",
      "        110,\r\n",
      "        100,\r\n",
      "        24,\r\n",
      "        22,\r\n",
      "        77\r\n",
      "      ]\r\n",
      "    },\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT1\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"parameters\": {},\r\n",
      "      \"data\": [\r\n",
      "        -77,\r\n",
      "        33,\r\n",
      "        25,\r\n",
      "        -52,\r\n",
      "        -49,\r\n",
      "        -88,\r\n",
      "        -48,\r\n",
      "        0,\r\n",
      "        -50,\r\n",
      "        26,\r\n",
      "        -44,\r\n",
      "        46,\r\n",
      "        -2,\r\n",
      "        18,\r\n",
      "        -6,\r\n",
      "        -47\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!cat /tmp/tfsimple-output.txt | head -n 1 | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d17f03fa-08bc-42aa-95e8-8427e609e045",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"\",\r\n",
      "  \"id\": \"46b05aab-07d9-414d-be96-c03d1863552a\",\r\n",
      "  \"parameters\": {\r\n",
      "    \"batch_index\": 3\r\n",
      "  },\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT0\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"data\": [\r\n",
      "        140,\r\n",
      "        164,\r\n",
      "        85,\r\n",
      "        58,\r\n",
      "        152,\r\n",
      "        76,\r\n",
      "        70,\r\n",
      "        56,\r\n",
      "        100,\r\n",
      "        141,\r\n",
      "        98,\r\n",
      "        181,\r\n",
      "        115,\r\n",
      "        177,\r\n",
      "        106,\r\n",
      "        193\r\n",
      "      ]\r\n",
      "    },\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT1\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"data\": [\r\n",
      "        -10,\r\n",
      "        0,\r\n",
      "        -11,\r\n",
      "        -38,\r\n",
      "        2,\r\n",
      "        -36,\r\n",
      "        -52,\r\n",
      "        -8,\r\n",
      "        -18,\r\n",
      "        57,\r\n",
      "        94,\r\n",
      "        -5,\r\n",
      "        -27,\r\n",
      "        17,\r\n",
      "        58,\r\n",
      "        -1\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!cat /tmp/tfsimple-pipeline-output.txt | head -n 1 | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd8d138-4825-4a06-a395-bfe332dbfb8e",
   "metadata": {},
   "source": [
    "## Cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b8d20578-5e4a-445a-a947-f09c8327033c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io \"iris\" deleted\n",
      "pipeline.mlops.seldon.io \"iris-pipeline\" deleted\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f models/sklearn-iris-gs.yaml -n ${NAMESPACE}\n",
    "!kubectl delete -f pipelines/iris.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5de7287b-577f-409f-af56-03aac9ec1f2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io \"tfsimple1\" deleted\n",
      "pipeline.mlops.seldon.io \"tfsimple\" deleted\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f models/tfsimple1.yaml -n ${NAMESPACE}\n",
    "!kubectl delete -f pipelines/tfsimple.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91bcec77",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
